import os
import json
import re

from module.spider import config
from module.spider import webspider

from module.ifile.fileutils import listAllFile

dirname, filename = os.path.split(os.path.abspath(__file__)) 
# os.path.realpath(__file__)

def clearEmptySpiderDataFile(folder=None):
    storePath = os.path.join(dirname, 'module/spider/store')

    if folder:
        storePath = os.path.join(storePath, folder)

    filenames = listAllFile(storePath, exclude=['.gitkeep', 'data.txt'])
    print(len(filenames))

    for filename in filenames:
        fd = open(filename, mode="r", encoding="utf-8")
        info = fd.read()
        fd.close()

        jsonData = json.loads(info) if len(info) else {'spiderData': {}}

        spiderData = jsonData['spiderData']
        if 'title' not in spiderData and 'content' not in spiderData:
            os.remove(filename)

    filenames = listAllFile(storePath, exclude=['.gitkeep', 'data.txt'])
    print(len(filenames))

def strsplit(value, splitchar):
    result = []
    if value:
        vArr = value.split(splitchar)
        if len(vArr) > 0:
            for item in vArr:
                item = item.strip()
                if item and item != '':
                    result.append(item)
    return result

def strHandle(value, splitchars):
    result = []
    splitchar = splitchars[0]
    del splitchars[0]
    nArr = strsplit(value, splitchar)
    if len(nArr) > 0:
        for n in nArr:
            if len(splitchars) == 0:
                result.append(n)
            else:
                result = result + strHandle(n, splitchars)
    return result

def handelData(folder=None):
    storePath = os.path.join(dirname, 'module/spider/store')

    if folder:
        storePath = os.path.join(storePath, folder)

    filenames = listAllFile(storePath, exclude=['.gitkeep', 'data.txt'])

    result = []

    for filename in filenames:
        fd = open(filename, mode="r", encoding="utf-8")
        info = fd.read()
        fd.close()

        try:
            jsonData = json.loads(info) if len(info) else {'spiderData': {}}
        except Exception as e:
            print(e, filename)

        spiderData = jsonData['spiderData']

        title = spiderData['title'] if 'title' in spiderData else ''
        content = spiderData['content'] if 'content' in spiderData else ''

        if title != '':
            # values = strHandle(title, ['\r', '\n', '\t', '。'])
            values = re.split('\r|\n|\t|。', title)
            for value in values:
                if value and value != '':
                    result.append(value)

        if content != '':
            # values = strHandle(content, ['\r', '\n', '\t', '。'])
            values = re.split('\r|\n|\t|。', content)
            for value in values:
                if value and value != '':
                    result.append(value)

    return result

def transforData(folder=None):
    datas = handelData(folder)

    storePath = os.path.join(dirname, 'module/spider/store')
    savePath = os.path.join(storePath, 'data.txt')

    fd = open(savePath, mode="w", encoding="utf-8")
    for line in datas:
        fd.write(line + '\n')
    fd.close()

    lines = []
    fd = open(savePath, mode="r", encoding="utf-8")
    for line in fd.readlines():
        line = line.strip()
        if line and line != '':
            lines.append(line)
    fd.close()

    lines = list(set(lines))
    fd = open(savePath, mode="w", encoding="utf-8")
    for line in lines:
        fd.write(line + '\n')
    fd.close()

import random
def randomUnicode():
    # 随机生成汉字
    val = random.randint(0x4e00, 0x9fbf)
    return chr(val)

def dropData():
    storePath = os.path.join(dirname, 'module/spider/store')
    savePath = os.path.join(storePath, 'data.txt')

    #定义标点替换为随机字符
    tempStr=''

    fd = open(savePath, mode="r", encoding="utf-8")
    for line in fd.readlines():
        line = line.strip()
        if line and line != '':
            values = re.split(',|，|\.|。|:|：|;|；|\?|？|!|！|、|"|\'|“|”|‘|’|\(|\)|（|）|\[|\]|【|】|「|」|…|@|\n|\u3000|\s', line)
            for value in values:
                if value and value != '':
                    tempStr += value + randomUnicode()
    fd.close()

    fd = open(savePath, mode="w", encoding="utf-8")
    fd.write(tempStr)
    fd.close()

def start():
    webspiderConfigs = config.listConfig()

    for webspiderConfig in webspiderConfigs:
        webspider.start(webspiderConfig)

# start()
# clearEmptySpiderDataFile('1/艺术')
# clearEmptySpiderDataFile('1/其他')
transforData('1/艺术')
dropData()